from langchain.document_loaders import DirectoryLoader, TextLoader, PythonLoader
import nltk
nltk.download('averaged_perceptron_tagger')


def normal_directory_loader():
    loader = DirectoryLoader("files", glob="**/*.md")
    mds = loader.load()
    print(mds)
    print(len(mds))


def directory_loader_with_progress():
    loader = DirectoryLoader("files", glob="**/*.md", show_progress=True)
    loader.load()


def directory_loader_with_multithreading():
    loader = DirectoryLoader("files", glob="**/*.md", show_progress=True, use_multithreading=True)
    loader.load()


def directory_loader_with_text_loader():
    loader = DirectoryLoader("files", glob="**/*.md", show_progress=True, use_multithreading=True, loader_cls=TextLoader)
    # 参数 silent_errors 传递给 DirectoryLoader，以跳过无法加载的文件并继续加载过程
    # 自动检查编码{'autodetect_encoding': True}
    loader = DirectoryLoader("files", glob="**/*.md", show_progress=True, use_multithreading=True,
                             loader_cls=TextLoader, silent_errors=True, loader_kwargs={'autodetect_encoding': True})
    docs = loader.load()
    doc_resource = [doc.metadata["source"] for doc in docs]
    for item in doc_resource:
        print(item)


def directory_loader_with_python_loader():
    loader = DirectoryLoader("../", glob="**/*.py", loader_cls=PythonLoader)
    docs = loader.load()
    print(docs)


if __name__ == '__main__':
    directory_loader_with_text_loader()